{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import os\n",
    "import json\n",
    "import random\n",
    "from datasets import load_dataset,concatenate_datasets,Dataset\n",
    "from tqdm import tqdm"
   ],
   "id": "a988ba7f28eb106a"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds=load_dataset('json', data_files=f'/home/kas/zhangyi/mrc.jsonl')",
   "id": "846a845ff39deb23"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "# ",
   "id": "dc1a26ba5458ed66"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "# ds['train'][100:]",
   "id": "1d472e0b8cfae3c3"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import pyarrow as pa\n",
    "from datasets import Dataset\n",
    "def read_arrow_to_df_julia_ok(path):\n",
    "    with open(path,\"rb\") as f:\n",
    "        r = pa.ipc.RecordBatchStreamReader(f)\n",
    "        df = r.read_pandas()\n",
    "    return df"
   ],
   "id": "679f3e420432083b"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import pyarrow as pa\n",
    "from datasets import Dataset\n",
    "def read_arrow_to_df_julia_ok(path):\n",
    "    r = pa.ipc.RecordBatchStreamReader(path)\n",
    "    df = r.read_pandas()\n",
    "    return df"
   ],
   "id": "e5dfa726cc3d7e69"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "df=read_arrow_to_df_julia_ok(\"/home/kas/.cache/huggingface/datasets/json/default-34dc6729b5b6e360/0.0.0/a3e658c4731e59120d44081ac10bf85dc7e1388126b92338344ce9661907f253/json-train.arrow\")",
   "id": "ebdfa68c7276cbcd"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "dataset = Dataset.from_pandas(df)",
   "id": "fe22223716dd4fc3"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "# for index, row in df.iterrows():\n",
    "#     print(row['prompt'], row['response'])\n",
    "#     break"
   ],
   "id": "994c52b481d11b8f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "# ds",
   "id": "6ae7daf6ad25190d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "dataset",
   "id": "a99fd9601870eb1e"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "with open('/home/kas/zhangyi/zh_helpfulness.json',encoding=\"utf-8\") as f:\n",
    "    datas1=json.load(f)"
   ],
   "id": "ec9804aba45bf0e7"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "with open('/home/kas/zhangyi/zh_honesty.json',encoding=\"utf-8\") as f:\n",
    "    datas2=json.load(f)"
   ],
   "id": "92812a0f400f9378"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datas1[0]",
   "id": "94dcd555ece6c00"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datas2[0]",
   "id": "227b03f93bab2dff"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "def extractor(text):\n",
    "    qa_pairs = text.split(\"<eoa>\")[:-1]\n",
    "    qs,ans=[],[]\n",
    "    for i in range(len(qa_pairs)):\n",
    "        qa_pairs[i] = qa_pairs[i].split(\"<eoh>\")\n",
    "        qa_pairs[i][0] = qa_pairs[i][0].strip().strip(\"[MOSS]: \").strip(\"[Human]: \").strip()\n",
    "        qa_pairs[i][1] = qa_pairs[i][1].strip().strip(\"[MOSS]: \").strip(\"[Human]: \").strip()\n",
    "        qs.append(qa_pairs[i][0])\n",
    "        ans.append(qa_pairs[i][1])\n",
    "    return qa_pairs"
   ],
   "id": "88d7d5bcb295809f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "inputs,outputs=[],[]\n",
    "for data in tqdm(datas1):\n",
    "    qa_pairs=extractor(data['plain_text'])\n",
    "    instruction=''.join([f\"\\n<用户>:{qa[0]}\\n<张仪>:{qa[1]}</s>\" for qa in qa_pairs[:-1]])\n",
    "    instruction+=f\"\\n<用户>:{qa_pairs[-1][0]}\"\n",
    "    inputs.append(instruction.lstrip(\"\\n<用户>:\"))\n",
    "    outputs.append(qa_pairs[-1][1])"
   ],
   "id": "db7160f298fe0ece"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "for data in tqdm(datas2):\n",
    "    qa_pairs=extractor(data['plain_text'])\n",
    "    instruction=''.join([f\"\\n<用户>:{qa[0]}\\n<张仪>:{qa[1]}</s>\" for qa in qa_pairs[:-1]])\n",
    "    instruction+=f\"\\n<用户>:{qa_pairs[-1][0]}\"\n",
    "    inputs.append(instruction.lstrip(\"\\n<用户>:\"))\n",
    "    outputs.append(qa_pairs[-1][1])"
   ],
   "id": "48962df946d7483d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "inputs[0]",
   "id": "ffd5c0f25a0cc50f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "outputs[0]",
   "id": "18a159119ed54337"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds7 = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs,\"input\": [\"\"]*len(outputs), })",
   "id": "186ba400130e8df3"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds8 = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs,\"input\": [\"\"]*len(outputs) })",
   "id": "186a5ec46eb8d268"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds7.save_to_disk(\"zh_honesty_helpfulness\")",
   "id": "6a0c34538291458"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "len(inputs),len(outputs)",
   "id": "ce7998387427d426"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "dataset6 = load_dataset(\"BelleGroup/multiturn_chat_0.8M\")\n",
    "datas6 = [\n",
    "    {\"instruction\": data[\"instruction\"]. \\\n",
    "        replace(\"Human\",\"</s>\"  + \"\\n<用户>\"). \\\n",
    "        replace(\"Assistant\", \"\\n<张仪>\"). \\\n",
    "        lstrip(\"</s>\").\\\n",
    "        lstrip(\"\\n<用户>:\").\\\n",
    "        rstrip(\"\\n<张仪>:\"),\n",
    "     \"input\": data[\"input\"], \"output\": data[\"output\"]}\n",
    "    for data in tqdm(dataset6[\"train\"])]"
   ],
   "id": "2602e05323033b63"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "# datas6[:10]",
   "id": "74d602d7d55ad097"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "from datasets import concatenate_datasets",
   "id": "a37211241ca92abb"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds=concatenate_datasets([ds8,ds7,ds8])",
   "id": "b6b542fa49bc9a64"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds",
   "id": "582b0b10a893ce2c"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "ds14 = load_dataset('json', data_files=f'/home/kas/zhangyi/mrc.jsonl')\n",
    "ds14 = ds14.rename_column(\"prompt\", \"instruction\")\n",
    "ds14 = ds14.rename_column(\"response\", \"output\")"
   ],
   "id": "21822dd440bf30e2"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds14",
   "id": "61c1218d13c8291f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds14.skip(37879)",
   "id": "8dc69750e545e707"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "data=[len(data['instruction']+data['output']) for data in tqdm(ds14['train']) if len(data['instruction']+data['output'])<=800]",
   "id": "819ee5481298bc45"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# 生成一组随机数据\n",
    "# data = np.random.normal(size=1000)\n",
    "\n",
    "# 绘制直方图\n",
    "plt.hist(data, bins=30)\n",
    "plt.show()\n",
    "\n",
    "# 绘制密度图\n",
    "plt.hist(data, bins=30, density=True)\n",
    "plt.show()\n",
    "\n",
    "# 绘制箱线图\n",
    "plt.boxplot(data)\n",
    "plt.show()\n",
    "# 绘制水平直方图\n",
    "plt.hist(data, bins=30, orientation='horizontal')\n",
    "plt.show()"
   ],
   "id": "e1ee77d7fe2a3470"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "!sudo apt-get install rar\n",
    "!rar x MMCU.rar"
   ],
   "id": "2d36bd1ba873f0d2"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "inputs,outputs=[],[]\n",
    "for i in range(4):\n",
    "    ds=load_dataset('json', data_files=f'/home/kas/data/JEC-QA/{i}.json')\n",
    "    for data in tqdm(ds['train']):\n",
    "#         print(data)\n",
    "        q=data[\"statement\"]+ \" \"+' '.join(item +\" \"+ data['option_list'][item] for item in ['A', 'B', 'C', 'D'])\n",
    "        a=data[\"answer\"]\n",
    "        if len(a)>1:\n",
    "            a=''.join(a)\n",
    "        elif len(a)==1:\n",
    "            a=f\"{a[0]} {data['option_list'][a[0]]}\"\n",
    "        else:\n",
    "            continue\n",
    "        inputs.append(q)\n",
    "        outputs.append(a)\n",
    "ds7 = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs})"
   ],
   "id": "9e69b8f5cf5acf8"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds7",
   "id": "a154934d0db30497"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds7[0]",
   "id": "4c65b9c56ca982ff"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds7.save_to_disk(\"JEC-QA-datasets\")",
   "id": "e8b1c1a741ebf94d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import pandas as pd\n",
    "import openpyxl"
   ],
   "id": "137d9a221dc77242"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "inputs,outputs=[],[]\n",
    "for f in os.listdir(\"MMCU0513/dev\"):\n",
    "    if f.split('.')[-1]!='xlsx':\n",
    "        continue\n",
    "    workbook = openpyxl.load_workbook(f'/home/kas/data/MMCU0513/dev/{f}')\n",
    "    data = worksheet.values\n",
    "    cols = next(data)\n",
    "    df = pd.DataFrame(data, columns=cols)\n",
    "    for row in df.iterrows():\n",
    "        if not row[1].题目:\n",
    "            continue\n",
    "        q=row[1].题目+ \" \"+' '.join(item+ \" \"+ row[1][f'选项{item}'] for item in ['A', 'B', 'C', 'D'])\n",
    "        a=row[1][\"正确答案\"]\n",
    "        if len(a)>1:\n",
    "            ...\n",
    "        elif len(a)==1:\n",
    "            a=f\"{a} {row[1][f'选项{a}']}\"\n",
    "        else:\n",
    "            continue\n",
    "        inputs.append(q)\n",
    "        outputs.append(a)"
   ],
   "id": "7fcff907624b2f9f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "for f in os.listdir(\"MMCU0513/test\"):\n",
    "    workbook = openpyxl.load_workbook(f'/home/kas/data/MMCU0513/test/{f}')\n",
    "    data = worksheet.values\n",
    "    cols = next(data)\n",
    "    df = pd.DataFrame(data, columns=cols)\n",
    "    for row in df.iterrows():\n",
    "        if not row[1].题目:\n",
    "            continue\n",
    "        q=row[1].题目+\" \"+' '.join(item + \" \"+row[1][f'选项{item}'] for item in ['A', 'B', 'C', 'D'])\n",
    "        a=row[1][\"正确答案\"]\n",
    "        if len(a)>1:\n",
    "            pass\n",
    "        elif len(a)==1:\n",
    "            a=f\"{a} {row[1][f'选项{a}']}\"\n",
    "        else:\n",
    "            continue\n",
    "        inputs.append(q)\n",
    "        outputs.append(a)\n",
    "ds8 = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs})"
   ],
   "id": "1cd88f0178fa62b5"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds8[0]",
   "id": "350f63011f2fd325"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds8",
   "id": "71206d61760fc9db"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds8.save_to_disk(\"MMCU0513-datasets\")",
   "id": "97f50f0187f73004"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "inputs,outputs=[],[]\n",
    "for f in os.listdir(\"/home/kas/data/AGIEval-main/v1/\"):\n",
    "    if f.split('.')[-1]!='jsonl':\n",
    "        continue\n",
    "    print(f)\n",
    "    try:\n",
    "        datas=load_dataset('json', data_files=f'/home/kas/data/AGIEval-main/v1/{f}')['train']\n",
    "    except:\n",
    "        with open(f'/home/kas/data/AGIEval-main/v1/{f}',encoding=\"utf-8\") as f:\n",
    "            datas=f.readlines()\n",
    "        \n",
    "    for data in tqdm(datas):\n",
    "            if type(data)==str:\n",
    "                data=json.loads(data)\n",
    "            q=data[\"question\"]+ \" \"+' '.join(data['options']) if data['options'] else data[\"question\"]\n",
    "            other=data.get('other',{}) if data.get('other') else {}\n",
    "            if data[\"label\"]:\n",
    "                label=''.join(data[\"label\"]) if type(data[\"label\"])==list else data[\"label\"]\n",
    "            else:\n",
    "                label=None\n",
    "            a=label+\" \\n\\n \"+other.get('solution',\"\")   if  label else data[\"answer\"]+\" \\n\\n \"+other.get('solution',\"\")\n",
    "            if len(a)>1:\n",
    "                a=''.join(a)\n",
    "            elif len(a)==1:\n",
    "                a=f\"{a[0]} {data['option_list'][a[0]]}\"\n",
    "            else:\n",
    "                continue\n",
    "            inputs.append(q)\n",
    "            outputs.append(a)\n",
    "ds9 = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs})"
   ],
   "id": "b947f7151e42e8b2"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds9.save_to_disk(\"AGIEval-datasets\")",
   "id": "656fa8b9a4dab161"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds9",
   "id": "375c7b90e87b38ed"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds9[0]",
   "id": "f8ce59a3d5badccc"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "5c3444de61472620"
  }
 ],
 "metadata": {},
 "nbformat": 5,
 "nbformat_minor": 9
}
